package com.feng.app.spider;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;

/**
 * 
 * 暂时不用线程
 * 
 * @author feng
 * 
 */
public class Get {
	HttpClient client = HttpClients.createDefault();
	String imaSrc="<img.*src=(.*?)[^>]*?>";
	String httpSrc= "http:\"?(.*?)(\"|>|\\s+)"; 
	/**
	 * 获取网页中所有图片的url
	 * 利用正则表达式
	 * @return
	 */
	public LinkedList<String> getImaUrls(String url) {
		LinkedList<String> urls = new LinkedList<String>();
		HttpGet get = new HttpGet(url.replace("\"", "").trim());
		
		 
		
		
		try {
			// 获取结果
			HttpResponse res = client.execute(get);
			// 写入实体
			HttpEntity entity = res.getEntity();
			// 获取文件流
			InputStream is;

			is = entity.getContent();

			InputStreamReader isr = new InputStreamReader(is);
			BufferedReader br = new BufferedReader(isr);
			String line;
			
			while((line=br.readLine())!=null){
				//正则
				Pattern p=Pattern.compile(imaSrc);
				Matcher m=p.matcher(line);
				if(m.find()){
					Pattern ps=Pattern.compile(httpSrc);
					Matcher ms=ps.matcher(m.group());
					
					System.out.println(m.group());
					
					if(ms.find()){
						//添加
						urls.add(ms.group().substring(0,ms.group().length()-1));
						System.out.println("----------------------"+ms.group());
					}
				}
				
			}

		} catch (UnsupportedOperationException | IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		return urls;
		
	}

	/**
	 * 获取网页中所有url
	 * 
	 * @return set《String》
	 */
	public LinkedList<String> getUrls(String url) {
		// 连接
		HttpGet get = new HttpGet(url.replace("\"", "").trim());
		LinkedList<String> urls = new LinkedList<String>();
		try {
			// 获取结果
			HttpResponse res = client.execute(get);
			// 写入实体
			HttpEntity entity = res.getEntity();
			// 获取文件流
			InputStream is = entity.getContent();

			InputStreamReader isr = new InputStreamReader(is);
			BufferedReader br = new BufferedReader(isr);
			String line;
			while ((line = br.readLine()) != null) {
				Pattern p=Pattern.compile(httpSrc);
				Matcher m=p.matcher(line);
				if(m.find()&&!m.group().contains("/www.w3.org/")){
					urls.add(m.group().substring(0,m.group().length()-1));
				}
				
			}
			br.close();
			isr.close();
			is.close();

		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return urls;
	}

}
